<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<!-- saved from url=(0042)http://www.manongjc.com/article/17612.html -->
<html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">

<title>在scrapy简单使用代理IP和随机的User-Agent - 码农教程</title>
<meta name="description" content="本文章向大家介绍在scrapy简单使用代理IP和随机的User-Agent，需要的朋友可以参考一下"><link rel="stylesheet" type="text/css" href="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/style.css">
<script src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f.txt"></script><script src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/osd.js.下载"></script><script src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(1).txt"></script><script src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(2).txt" id="google_shimpl"></script><script src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/push.js.下载"></script><script src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/hm.js.下载"></script><script type="text/javascript" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/jquery-1.7.1.min.js.下载"></script>
<script type="text/javascript" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/static.js.下载"></script>
<script type="text/javascript" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/highlight.js.下载"></script>
	<link rel="stylesheet" type="text/css" href="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/xcode.css">
	<script>hljs.initHighlightingOnLoad();</script><script async="" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(3).txt"></script>
<link rel="shortcut icon" type="image/x-icon" href="http://www.manongjc.com/favicon.ico" media="screen">
<meta http-equiv="origin-trial" content="A88otRz1Fd3Nt567e2IYshC18LL3KGVXpVJW9oTCId4RYaygt23pbb4JqrbdIO/bwZPWEmRjBIRBu/bZbDR7Pg4AAABueyJvcmlnaW4iOiJodHRwczovL2ltYXNkay5nb29nbGVhcGlzLmNvbTo0NDMiLCJmZWF0dXJlIjoiVHJ1c3RUb2tlbnMiLCJleHBpcnkiOjE2MzQwODMxOTksImlzVGhpcmRQYXJ0eSI6dHJ1ZX0="><meta http-equiv="origin-trial" content="A0gCLbXCcL0R1Oc8tFPDs0G4Elz17w3zHp+Zst66+D17veE2o7fUcPsA114QtSTRqfVJLMeTSdeWOom0CcyCsgYAAAB7eyJvcmlnaW4iOiJodHRwczovL2RvdWJsZWNsaWNrLm5ldDo0NDMiLCJmZWF0dXJlIjoiVHJ1c3RUb2tlbnMiLCJleHBpcnkiOjE2MzQwODMxOTksImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9"><meta http-equiv="origin-trial" content="A9RQ+LxFazAousxUwSCzaihJjHLO1UyjQp0teZKHl7WdbVjPDfHSKMd6D/ZI5MTjqClFycbl70EFd7cBJWXqKQEAAACBeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXRhZ3NlcnZpY2VzLmNvbTo0NDMiLCJmZWF0dXJlIjoiVHJ1c3RUb2tlbnMiLCJleHBpcnkiOjE2MzQwODMxOTksImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9"><meta http-equiv="origin-trial" content="A6WKeWsdn1Ct+ZPqS9NCxxaiBoQ7wdTkK2/gE69Yu0gfBKJfo1gOvgkGmf5/xaIajT/RUb9AbnF1FsSZ47cCcQcAAACBeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXN5bmRpY2F0aW9uLmNvbTo0NDMiLCJmZWF0dXJlIjoiVHJ1c3RUb2tlbnMiLCJleHBpcnkiOjE2MzQwODMxOTksImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9"><meta http-equiv="origin-trial" content="A04ZCu7yjrHgwQJK5ISHhH1DSg0qqowEay3n70KO6wV3D2Mj+OX3Kw20aSMitzgdG1xfrN7sOJV/dZIk+RvCzA4AAAB2eyJvcmlnaW4iOiJodHRwczovL2dvb2dsZS5jb206NDQzIiwiZmVhdHVyZSI6IlRydXN0VG9rZW5zIiwiZXhwaXJ5IjoxNjM0MDgzMTk5LCJpc1N1YmRvbWFpbiI6dHJ1ZSwiaXNUaGlyZFBhcnR5Ijp0cnVlfQ=="><meta http-equiv="origin-trial" content="A88otRz1Fd3Nt567e2IYshC18LL3KGVXpVJW9oTCId4RYaygt23pbb4JqrbdIO/bwZPWEmRjBIRBu/bZbDR7Pg4AAABueyJvcmlnaW4iOiJodHRwczovL2ltYXNkay5nb29nbGVhcGlzLmNvbTo0NDMiLCJmZWF0dXJlIjoiVHJ1c3RUb2tlbnMiLCJleHBpcnkiOjE2MzQwODMxOTksImlzVGhpcmRQYXJ0eSI6dHJ1ZX0="><meta http-equiv="origin-trial" content="A0gCLbXCcL0R1Oc8tFPDs0G4Elz17w3zHp+Zst66+D17veE2o7fUcPsA114QtSTRqfVJLMeTSdeWOom0CcyCsgYAAAB7eyJvcmlnaW4iOiJodHRwczovL2RvdWJsZWNsaWNrLm5ldDo0NDMiLCJmZWF0dXJlIjoiVHJ1c3RUb2tlbnMiLCJleHBpcnkiOjE2MzQwODMxOTksImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9"><meta http-equiv="origin-trial" content="A9RQ+LxFazAousxUwSCzaihJjHLO1UyjQp0teZKHl7WdbVjPDfHSKMd6D/ZI5MTjqClFycbl70EFd7cBJWXqKQEAAACBeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXRhZ3NlcnZpY2VzLmNvbTo0NDMiLCJmZWF0dXJlIjoiVHJ1c3RUb2tlbnMiLCJleHBpcnkiOjE2MzQwODMxOTksImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9"><meta http-equiv="origin-trial" content="A6WKeWsdn1Ct+ZPqS9NCxxaiBoQ7wdTkK2/gE69Yu0gfBKJfo1gOvgkGmf5/xaIajT/RUb9AbnF1FsSZ47cCcQcAAACBeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXN5bmRpY2F0aW9uLmNvbTo0NDMiLCJmZWF0dXJlIjoiVHJ1c3RUb2tlbnMiLCJleHBpcnkiOjE2MzQwODMxOTksImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9"><meta http-equiv="origin-trial" content="A04ZCu7yjrHgwQJK5ISHhH1DSg0qqowEay3n70KO6wV3D2Mj+OX3Kw20aSMitzgdG1xfrN7sOJV/dZIk+RvCzA4AAAB2eyJvcmlnaW4iOiJodHRwczovL2dvb2dsZS5jb206NDQzIiwiZmVhdHVyZSI6IlRydXN0VG9rZW5zIiwiZXhwaXJ5IjoxNjM0MDgzMTk5LCJpc1N1YmRvbWFpbiI6dHJ1ZSwiaXNUaGlyZFBhcnR5Ijp0cnVlfQ=="><meta http-equiv="origin-trial" content="A2YAd4xOntTGygIDjApOTtXOgVI3IWsd5OnOGq3RbRkIQwyqYWNl1JGRAcvtm6VOHDj4n07T/J19VqLuJn3MmQ8AAACWeyJvcmlnaW4iOiJodHRwczovL2RvdWJsZWNsaWNrLm5ldDo0NDMiLCJmZWF0dXJlIjoiQ29udmVyc2lvbk1lYXN1cmVtZW50IiwiZXhwaXJ5IjoxNjMxNjYzOTk5LCJpc1N1YmRvbWFpbiI6dHJ1ZSwiaXNUaGlyZFBhcnR5Ijp0cnVlLCJ1c2FnZSI6InN1YnNldCJ9"><meta http-equiv="origin-trial" content="A2c5Ux+hivdkLh/KbZUGr6f7SCR0mZrBVfPJ+/OuDVHNwiYv+Lo83b9z5qL8sod78bQl0pSLtbvRWURo+xRl7AIAAACceyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXN5bmRpY2F0aW9uLmNvbTo0NDMiLCJmZWF0dXJlIjoiQ29udmVyc2lvbk1lYXN1cmVtZW50IiwiZXhwaXJ5IjoxNjMxNjYzOTk5LCJpc1N1YmRvbWFpbiI6dHJ1ZSwiaXNUaGlyZFBhcnR5Ijp0cnVlLCJ1c2FnZSI6InN1YnNldCJ9"><meta http-equiv="origin-trial" content="AzNJ4sd3tVurolpdvWYZ4cmP9Po7RJhEHSqmC3pgxW9fFVZvchhtcMUgHAs97npxMD1jhXHO8s6q6Wy1MMLxKgEAAACceyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXRhZ3NlcnZpY2VzLmNvbTo0NDMiLCJmZWF0dXJlIjoiQ29udmVyc2lvbk1lYXN1cmVtZW50IiwiZXhwaXJ5IjoxNjMxNjYzOTk5LCJpc1N1YmRvbWFpbiI6dHJ1ZSwiaXNUaGlyZFBhcnR5Ijp0cnVlLCJ1c2FnZSI6InN1YnNldCJ9"><meta http-equiv="origin-trial" content="AwfG8hAcHnPa/kJ1Co0EvG/K0F9l1s2JZGiDLt2mhC3QI5Fh4qmsmSwrWObZFbRC9ieDaSLU6lHRxhGUF/i9sgoAAACBeyJvcmlnaW4iOiJodHRwczovL2RvdWJsZWNsaWNrLm5ldDo0NDMiLCJmZWF0dXJlIjoiSW50ZXJlc3RDb2hvcnRBUEkiLCJleHBpcnkiOjE2MjYyMjA3OTksImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9"><meta http-equiv="origin-trial" content="AwQ7dCmHkvR6FuOFxAuNnktYSQrGbL4dF+eBkrwNLALc69Wr//PnO1yzns3pjUoCaYbKHtVcnng2hU+8OUm0PAYAAACHeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXN5bmRpY2F0aW9uLmNvbTo0NDMiLCJmZWF0dXJlIjoiSW50ZXJlc3RDb2hvcnRBUEkiLCJleHBpcnkiOjE2MjYyMjA3OTksImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9"><meta http-equiv="origin-trial" content="AysVDPGQTLD/Scn78x4mLwB1tMfje5jwUpAAzGRpWsr1NzoN7MTFhT3ClmImi2svDZA7V6nWGIV8YTPsSRTe0wYAAACHeyJvcmlnaW4iOiJodHRwczovL2dvb2dsZXRhZ3NlcnZpY2VzLmNvbTo0NDMiLCJmZWF0dXJlIjoiSW50ZXJlc3RDb2hvcnRBUEkiLCJleHBpcnkiOjE2MjYyMjA3OTksImlzU3ViZG9tYWluIjp0cnVlLCJpc1RoaXJkUGFydHkiOnRydWV9"><link rel="preload" href="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(4).txt" as="script"><script type="text/javascript" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(4).txt"></script><link rel="preload" href="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(4).txt" as="script"><script type="text/javascript" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(4).txt"></script><link rel="preload" href="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(4).txt" as="script"><script type="text/javascript" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(4).txt"></script></head>
<body aria-hidden="false">
<div class="page-content">
	<div class="header">
		<div class="logo">
			<a href="http://www.manongjc.com/"><img alt="码农教程,码农编程入门网" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/logo.gif"></a>
		</div>
		<div class="header-ads">
			<a href="https://www.aliyun.com/minisite/goods?userCode=da65xs35" target="_BLANK"><img src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/468-60.jpg"></a>
		</div>
		<div class="search">
			<div class="search-contain">
			<div id="bdcs">
	<div class="bdcs-container">
		<meta http-equiv="x-ua-compatible" content="IE=9">   <!-- 嵌入式 -->  
		<div class="bdcs-main bdcs-clearfix" id="default-searchbox">      
			<div class="bdcs-search bdcs-clearfix" id="bdcs-search-inline">          
				<form action="http://zhannei.baidu.com/cse/search" method="get" target="_blank" class="bdcs-search-form" autocomplete="off" id="bdcs-search-form">              
					<input type="hidden" name="s" value="1873544121090210198">              
					<input type="hidden" name="entry" value="1">                                                                        
					<input type="text" name="q" class="bdcs-search-form-input" id="bdcs-search-form-input" placeholder="请输入关键词" style="height: 34px; line-height: 34px;">              
					<input type="submit" class="bdcs-search-form-submit " id="bdcs-search-form-submit" value="搜索">                       
				</form>      
			</div>                  
		</div>                           
	</div>
</div>
			</div>
		</div>
		<div class="break"> </div>
	</div>
	<div class="menu">
		<ul>
			<li><a href="http://www.manongjc.com/" class="hide first"><b>首页</b></a>
			</li>
			<li><a href="http://www.manongjc.com/category/1.html" class="hide"><b>PHP</b></a>
			</li>
            <li><a href="http://www.manongjc.com/category/5.html" class="hide"><b>JAVA</b></a>
			</li>
            <li><a href="http://www.manongjc.com/category/11.html" class="hide"><b>C/C++</b></a>
			</li>
            <li><a href="http://www.manongjc.com/category/8.html" class="hide"><b>PYTHON</b></a>
			</li>
            <li><a href="http://www.manongjc.com/category/3.html" class="hide"><b>HTML/CSS</b></a>
			</li>
            <li><a href="http://www.manongjc.com/category/4.html" class="hide"><b>JS/JQUERY</b></a>
			</li>
            <li><a href="http://www.manongjc.com/category/2.html" class="hide"><b>MYSQL</b></a>
			</li>
		</ul>
		<div class="break"> </div>
	</div>
	<div class="breadcrumbs"><a href="http://www.manongjc.com/" class="title">首页</a>&nbsp;&gt;&nbsp;<a href="http://www.manongjc.com/biji" class="title">编程笔记</a>&nbsp;&gt;&nbsp;<a class="title" href="http://www.manongjc.com/category/8.html">python</a>&nbsp;&gt;&nbsp;在scrapy简单使用代理IP和随机的User-Agent</div>
	<div class="main-content">
		<div class="left">
			<div class="first-left">
				<div class="left-content">
					<div class="article-title"><h1>在scrapy简单使用代理IP和随机的User-Agent</h1></div>
					<div class="info">时间:2018-11-07					</div>
					<div class="intro">本文章向大家介绍在scrapy简单使用代理IP和随机的User-Agent，需要的朋友可以参考一下</div>					<div class="article-content" id="code_example">
						<div id="article_left_top_banner"><!-- article_left_top_banner -->
<ins class="adsbygoogle common_ad_class" style="display: inline-block; width: 600px; height: 300px;" data-ad-client="ca-pub-9401041403549801" data-ad-slot="1516503570" data-adsbygoogle-status="done" data-ad-status="filled"><ins id="aswift_0_expand" style="display:inline-table;border:none;height:300px;margin:0;padding:0;position:relative;visibility:visible;width:600px;background-color:transparent;" tabindex="0" title="Advertisement" aria-label="Advertisement"><ins id="aswift_0_anchor" style="display:block;border:none;height:300px;margin:0;padding:0;position:relative;visibility:visible;width:600px;background-color:transparent;"><iframe id="aswift_0" name="aswift_0" style="left:0;position:absolute;top:0;border:0;width:600px;height:300px;" sandbox="allow-forms allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-top-navigation-by-user-activation" width="600" height="300" frameborder="0" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/ads.html" marginwidth="0" marginheight="0" vspace="0" hspace="0" allowtransparency="true" scrolling="no" allowfullscreen="true" data-google-container-id="a!1" data-google-query-id="CKvL4pvY4fICFWCSSwUda4wP9A" data-load-complete="true"></iframe></ins></ins></ins></div>
<script>
if(document.getElementById("article_left_top_banner").offsetParent !== null){
	(adsbygoogle = window.adsbygoogle || []).push({});
}
</script>                        <p><span style="font-size:14pt">在使用爬虫的时候，有时候会看到<span style="font-size:18pt"><strong>由于目标计算机积极拒绝，无法连接...</strong><span style="font-size:14pt">，这就是因为我们的爬虫被识别出来了，而这种反爬虫主要是通过IP识别的，针对这种反爬虫，我们可以搭建一个自己的IP代理池，然后用这些IP去访问网站。</span></span></span></p>

<p><span style="font-size:14pt">新建一个proxies.py，实现我们自己的代理池，主要模块有获取模块、验证模块和存储模块。</span></p>

<p><span style="font-size:14pt">（1）获取模块：爬取网上的免费代理，这一部分我就写了个爬取西刺代理的代码，除了获取网上的代理，还要对代理池中代理进行清洗，因为有些代理会失效；</span></p>

<p><span style="font-size:14pt">（2）验证模块：验证我们的代理是否可用，如果不可用就舍弃掉；</span></p>

<p><span style="font-size:14pt">（3）存储模块：这一部分写得很简单，就是把可用的代理存储到proxies.txt中。</span></p>

<p><span style="font-size:14pt">具体代码如下：</span></p>

<div class="cnblogs_code">
<pre><span style="color:#008080"> 1</span> <span style="color:#0000ff">import</span><span style="color:#000000"> requests
</span><span style="color:#008080"> 2</span> <span style="color:#0000ff">import</span><span style="color:#000000"> re
</span><span style="color:#008080"> 3</span> 
<span style="color:#008080"> 4</span> 
<span style="color:#008080"> 5</span> <span style="color:#0000ff">class</span><span style="color:#000000"> Proxies:
</span><span style="color:#008080"> 6</span>     <span style="color:#0000ff">def</span> <span style="color:#800080">__init__</span><span style="color:#000000">(self):
</span><span style="color:#008080"> 7</span>         self.proxy_list =<span style="color:#000000"> []
</span><span style="color:#008080"> 8</span>         self.headers =<span style="color:#000000"> {
</span><span style="color:#008080"> 9</span>             <span style="color:#800000">"</span><span style="color:#800000">User-Agent</span><span style="color:#800000">"</span>: <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) </span><span style="color:#800000">"</span>
<span style="color:#008080">10</span>                           <span style="color:#800000">"</span><span style="color:#800000">Chrome/45.0.2454.101 Safari/537.36</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">11</span>             <span style="color:#800000">'</span><span style="color:#800000">Accept-Encoding</span><span style="color:#800000">'</span>: <span style="color:#800000">'</span><span style="color:#800000">gzip, deflate, sdch</span><span style="color:#800000">'</span><span style="color:#000000">,
</span><span style="color:#008080">12</span> <span style="color:#000000">        }
</span><span style="color:#008080">13</span> 
<span style="color:#008080">14</span>     <span style="color:#008000">#</span><span style="color:#008000"> 爬取西刺代理的国内高匿代理</span>
<span style="color:#008080">15</span>     <span style="color:#0000ff">def</span><span style="color:#000000"> get_proxy_nn(self):
</span><span style="color:#008080">16</span>         proxy_list =<span style="color:#000000"> []
</span><span style="color:#008080">17</span>         res = requests.get(<span style="color:#800000">"</span><span style="color:#800000">http://www.xicidaili.com/nn</span><span style="color:#800000">"</span>, headers=<span style="color:#000000">self.headers)
</span><span style="color:#008080">18</span>         ip_list = re.findall(<span style="color:#800000">'</span><span style="color:#800000">&lt;td&gt;(\d+\.\d+\.\d+\.\d+)&lt;/td&gt;</span><span style="color:#800000">'</span><span style="color:#000000">, res.text)
</span><span style="color:#008080">19</span>         port_list = re.findall(<span style="color:#800000">'</span><span style="color:#800000">&lt;td&gt;(\d+)&lt;/td&gt;</span><span style="color:#800000">'</span><span style="color:#000000">, res.text)
</span><span style="color:#008080">20</span>         <span style="color:#0000ff">for</span> ip, port <span style="color:#0000ff">in</span><span style="color:#000000"> zip(ip_list, port_list):
</span><span style="color:#008080">21</span>             proxy_list.append(ip + <span style="color:#800000">"</span><span style="color:#800000">:</span><span style="color:#800000">"</span> +<span style="color:#000000"> port)
</span><span style="color:#008080">22</span>         <span style="color:#0000ff">return</span><span style="color:#000000"> proxy_list
</span><span style="color:#008080">23</span> 
<span style="color:#008080">24</span>     <span style="color:#008000">#</span><span style="color:#008000"> 验证代理是否能用</span>
<span style="color:#008080">25</span>     <span style="color:#0000ff">def</span><span style="color:#000000"> verify_proxy(self, proxy_list):
</span><span style="color:#008080">26</span>         <span style="color:#0000ff">for</span> proxy <span style="color:#0000ff">in</span><span style="color:#000000"> proxy_list:
</span><span style="color:#008080">27</span>             proxies =<span style="color:#000000"> {
</span><span style="color:#008080">28</span>                 <span style="color:#800000">"</span><span style="color:#800000">http</span><span style="color:#800000">"</span><span style="color:#000000">: proxy
</span><span style="color:#008080">29</span> <span style="color:#000000">            }
</span><span style="color:#008080">30</span>             <span style="color:#0000ff">try</span><span style="color:#000000">:
</span><span style="color:#008080">31</span>                 <span style="color:#0000ff">if</span> requests.get(<span style="color:#800000">'</span><span style="color:#800000">http://www.baidu.com</span><span style="color:#800000">'</span>, proxies=proxies, timeout=2).status_code == 200<span style="color:#000000">:
</span><span style="color:#008080">32</span>                     <span style="color:#0000ff">print</span>(<span style="color:#800000">'</span><span style="color:#800000">success %s</span><span style="color:#800000">'</span> %<span style="color:#000000"> proxy)
</span><span style="color:#008080">33</span>                     <span style="color:#0000ff">if</span> proxy <span style="color:#0000ff">not</span> <span style="color:#0000ff">in</span><span style="color:#000000"> self.proxy_list:
</span><span style="color:#008080">34</span> <span style="color:#000000">                        self.proxy_list.append(proxy)
</span><span style="color:#008080">35</span>             <span style="color:#0000ff">except</span><span style="color:#000000">:
</span><span style="color:#008080">36</span>                 <span style="color:#0000ff">print</span>(<span style="color:#800000">'</span><span style="color:#800000">fail %s</span><span style="color:#800000">'</span> %<span style="color:#000000"> proxy)
</span><span style="color:#008080">37</span> 
<span style="color:#008080">38</span>     <span style="color:#008000">#</span><span style="color:#008000"> 保存到proxies.txt里</span>
<span style="color:#008080">39</span>     <span style="color:#0000ff">def</span><span style="color:#000000"> save_proxy(self):
</span><span style="color:#008080">40</span>         <span style="color:#008000">#</span><span style="color:#008000"> 验证代理池中的IP是否可用</span>
<span style="color:#008080">41</span>         <span style="color:#0000ff">print</span>(<span style="color:#800000">"</span><span style="color:#800000">开始清洗代理池...</span><span style="color:#800000">"</span><span style="color:#000000">)
</span><span style="color:#008080">42</span>         with open(<span style="color:#800000">"</span><span style="color:#800000">proxies.txt</span><span style="color:#800000">"</span>, <span style="color:#800000">'</span><span style="color:#800000">r</span><span style="color:#800000">'</span>, encoding=<span style="color:#800000">"</span><span style="color:#800000">utf-8</span><span style="color:#800000">"</span><span style="color:#000000">) as f:
</span><span style="color:#008080">43</span>             txt =<span style="color:#000000"> f.read()
</span><span style="color:#008080">44</span>         <span style="color:#008000">#</span><span style="color:#008000"> 判断代理池是否为空</span>
<span style="color:#008080">45</span>         <span style="color:#0000ff">if</span> txt != <span style="color:#800000">''</span><span style="color:#000000">:
</span><span style="color:#008080">46</span>             self.verify_proxy(txt.strip().split(<span style="color:#800000">'</span><span style="color:#800000">\n</span><span style="color:#800000">'</span><span style="color:#000000">))
</span><span style="color:#008080">47</span>         <span style="color:#0000ff">else</span><span style="color:#000000">:
</span><span style="color:#008080">48</span>             <span style="color:#0000ff">print</span>(<span style="color:#800000">"</span><span style="color:#800000">代理池为空！\n</span><span style="color:#800000">"</span><span style="color:#000000">)
</span><span style="color:#008080">49</span>         <span style="color:#0000ff">print</span>(<span style="color:#800000">"</span><span style="color:#800000">开始存入代理池...</span><span style="color:#800000">"</span><span style="color:#000000">)
</span><span style="color:#008080">50</span>         <span style="color:#008000">#</span><span style="color:#008000"> 把可用的代理添加到代理池中</span>
<span style="color:#008080">51</span>         with open(<span style="color:#800000">"</span><span style="color:#800000">proxies.txt</span><span style="color:#800000">"</span>, <span style="color:#800000">'</span><span style="color:#800000">w</span><span style="color:#800000">'</span>, encoding=<span style="color:#800000">"</span><span style="color:#800000">utf-8</span><span style="color:#800000">"</span><span style="color:#000000">) as f:
</span><span style="color:#008080">52</span>             <span style="color:#0000ff">for</span> proxy <span style="color:#0000ff">in</span><span style="color:#000000"> self.proxy_list:
</span><span style="color:#008080">53</span>                 f.write(proxy + <span style="color:#800000">"</span><span style="color:#800000">\n</span><span style="color:#800000">"</span><span style="color:#000000">)
</span><span style="color:#008080">54</span> 
<span style="color:#008080">55</span> 
<span style="color:#008080">56</span> <span style="color:#0000ff">if</span> <span style="color:#800080">__name__</span> == <span style="color:#800000">'</span><span style="color:#800000">__main__</span><span style="color:#800000">'</span><span style="color:#000000">:
</span><span style="color:#008080">57</span>     p =<span style="color:#000000"> Proxies()
</span><span style="color:#008080">58</span>     results =<span style="color:#000000"> p.get_proxy_nn()
</span><span style="color:#008080">59</span>     <span style="color:#0000ff">print</span>(<span style="color:#800000">"</span><span style="color:#800000">爬取到的代理数量</span><span style="color:#800000">"</span><span style="color:#000000">, len(results))
</span><span style="color:#008080">60</span>     <span style="color:#0000ff">print</span>(<span style="color:#800000">"</span><span style="color:#800000">开始验证：</span><span style="color:#800000">"</span><span style="color:#000000">)
</span><span style="color:#008080">61</span> <span style="color:#000000">    p.verify_proxy(results)
</span><span style="color:#008080">62</span>     <span style="color:#0000ff">print</span>(<span style="color:#800000">"</span><span style="color:#800000">验证完毕：</span><span style="color:#800000">"</span><span style="color:#000000">)
</span><span style="color:#008080">63</span>     <span style="color:#0000ff">print</span>(<span style="color:#800000">"</span><span style="color:#800000">可用代理数量：</span><span style="color:#800000">"</span><span style="color:#000000">, len(p.proxy_list))
</span><span style="color:#008080">64</span>     p.save_proxy()</pre>
</div>

<p><span style="font-size:14pt">运行完后会自动生成一个proxies.txt（如果没有proxies.txt这个文件），打开后可以看到一些可用的代理：</span></p>

<p><span style="font-size:14pt"><img alt="" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/15415786121541TA57S86612.png"></span></p>

<p><span style="font-size:14pt">但是这样只是搭建好了我们的代理池，要想使用的话，还要在middleware.py中添加一个类来使用我们的代理IP：</span></p>

<div class="cnblogs_code">
<pre><span style="color:#008080"> 1</span> <span style="color:#0000ff">class</span><span style="color:#000000"> NovelProxyMiddleWare(object):
</span><span style="color:#008080"> 2</span>     <span style="color:#0000ff">def</span><span style="color:#000000"> process_request(self, request, spider):
</span><span style="color:#008080"> 3</span>         proxy =<span style="color:#000000"> self.get_random_proxy()
</span><span style="color:#008080"> 4</span>         <span style="color:#0000ff">print</span>(<span style="color:#800000">"</span><span style="color:#800000">Request proxy is {}</span><span style="color:#800000">"</span><span style="color:#000000">.format(proxy))
</span><span style="color:#008080"> 5</span>         request.meta[<span style="color:#800000">"</span><span style="color:#800000">proxy</span><span style="color:#800000">"</span>] = <span style="color:#800000">"</span><span style="color:#800000">http://</span><span style="color:#800000">"</span> +<span style="color:#000000"> proxy
</span><span style="color:#008080"> 6</span> 
<span style="color:#008080"> 7</span>     <span style="color:#0000ff">def</span><span style="color:#000000"> get_random_proxy(self):
</span><span style="color:#008080"> 8</span>         <span style="color:#0000ff">import</span><span style="color:#000000"> random
</span><span style="color:#008080"> 9</span>         with open(proxies.txt的绝对路径, <span style="color:#800000">'</span><span style="color:#800000">r</span><span style="color:#800000">'</span>, encoding=<span style="color:#800000">"</span><span style="color:#800000">utf-8</span><span style="color:#800000">"</span><span style="color:#000000">) as f:
</span><span style="color:#008080">10</span>             txt =<span style="color:#000000"> f.read()
</span><span style="color:#008080">11</span>         <span style="color:#0000ff">return</span> random.choice(txt.split(<span style="color:#800000">'</span><span style="color:#800000">\n</span><span style="color:#800000">'</span>))</pre>
</div>

<p><span style="font-size:14pt">然后还要修改settings.py中的代码：</span></p>

<div class="cnblogs_code">
<pre><span style="color:#008080">1</span> DOWNLOADER_MIDDLEWARES =<span style="color:#000000"> {
</span><span style="color:#008080">2</span>     <span style="color:#800000">'</span><span style="color:#800000">Novels.middlewares.NovelProxyMiddleWare</span><span style="color:#800000">'</span>: 543<span style="color:#000000">,
</span><span style="color:#008080">3</span> }</pre>
</div>

<p><span style="font-size:14pt">进行完上述步骤后就能在我们的爬虫中使用代理IP了，如果还想在Scrapy中使用随机User-Agent的话，可以接着往下看。</span></p>

<p><span style="font-size:14pt">首先是在middleware.py中添加如下代码：</span></p>

<div class="cnblogs_code">
<pre><span style="color:#008080"> 1</span> <span style="color:#0000ff">class</span><span style="color:#000000"> NovelUserAgentMiddleWare(object):
</span><span style="color:#008080"> 2</span>     <span style="color:#0000ff">def</span> <span style="color:#800080">__init__</span><span style="color:#000000">(self):
</span><span style="color:#008080"> 3</span>         self.user_agent_list =<span style="color:#000000"> [
</span><span style="color:#008080"> 4</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1</span><span style="color:#800000">"</span>
<span style="color:#008080"> 5</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080"> 6</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080"> 7</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080"> 8</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080"> 9</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">10</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">11</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">12</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">13</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">14</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">15</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">16</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">17</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">18</span>             <span style="color:#800000">"</span><span style="color:#800000">Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24</span><span style="color:#800000">"</span><span style="color:#000000">,
</span><span style="color:#008080">19</span> <span style="color:#000000">        ]
</span><span style="color:#008080">20</span> 
<span style="color:#008080">21</span>     <span style="color:#0000ff">def</span><span style="color:#000000"> process_request(self, request, spider):
</span><span style="color:#008080">22</span>         ua =<span style="color:#000000"> random.choice(self.user_agent_list)
</span><span style="color:#008080">23</span>         <span style="color:#0000ff">print</span>(<span style="color:#800000">'</span><span style="color:#800000">User-Agent:</span><span style="color:#800000">'</span> +<span style="color:#000000"> ua)
</span><span style="color:#008080">24</span>         request.headers.setdefault(<span style="color:#800000">'</span><span style="color:#800000">User-Agent</span><span style="color:#800000">'</span>, ua)</pre>
</div>

<p><span style="font-size:14pt">然后修改settings.py中的内容：</span></p>

<div class="cnblogs_code">
<pre><span style="color:#008080">1</span> DOWNLOADER_MIDDLEWARES =<span style="color:#000000"> {
</span><span style="color:#008080">2</span>     <span style="color:#800000">'</span><span style="color:#800000">Novels.middlewares.NovelUserAgentMiddleWare</span><span style="color:#800000">'</span>: 544<span style="color:#000000">,
</span><span style="color:#008080">3</span>     <span style="color:#800000">'</span><span style="color:#800000">Novels.middlewares.NovelProxyMiddleWare</span><span style="color:#800000">'</span>: 543<span style="color:#000000">,
</span><span style="color:#008080">4</span> }</pre>
</div>

<p><span style="font-size:14pt">这样我们就能在Scrapy中使用代理IP和随机User-Agent了。当然了，由于使用的是免费代理，很可能短时间内就会失效了，可以考虑使用付费代理，效果会好很多。</span></p>						<div id="article_left_bottom_banner"><!-- article_left_bottom_banner -->
<ins class="adsbygoogle common_ad_class" style="display: inline-block; width: 600px; height: 300px;" data-ad-client="ca-pub-9401041403549801" data-ad-slot="4469969972" data-adsbygoogle-status="done" data-ad-status="filled"><ins id="aswift_1_expand" style="display:inline-table;border:none;height:300px;margin:0;padding:0;position:relative;visibility:visible;width:600px;background-color:transparent;" tabindex="0" title="Advertisement" aria-label="Advertisement"><ins id="aswift_1_anchor" style="display:block;border:none;height:300px;margin:0;padding:0;position:relative;visibility:visible;width:600px;background-color:transparent;"><iframe id="aswift_1" name="aswift_1" style="left:0;position:absolute;top:0;border:0;width:600px;height:300px;" sandbox="allow-forms allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-top-navigation-by-user-activation" width="600" height="300" frameborder="0" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/ads(1).html" marginwidth="0" marginheight="0" vspace="0" hspace="0" allowtransparency="true" scrolling="no" allowfullscreen="true" data-google-container-id="a!2" data-google-query-id="CPKvt6DY4fICFRqfSwUdGO8M0g" data-load-complete="true"></iframe></ins></ins></ins></div>
<script>
if(document.getElementById("article_left_bottom_banner").offsetParent !== null){
	(adsbygoogle = window.adsbygoogle || []).push({});
}
</script>                        <div class="break"></div>
                    </div>
				</div>
			</div>
			<div class="common">
    <div class="title-bar">热门文章</div>
	<div class="hot_article">
		<div class="hot_article_list">
			<ul>
				<li><a href="http://www.manongjc.com/article/89833.html">Python标准库映射类型与可散列数据类型的关系</a></li>
				<li><a href="http://www.manongjc.com/article/89769.html">C#中的参数和调用方式（可选参数、具名参数、可空参数）</a></li>
				<li><a href="http://www.manongjc.com/article/89662.html">Android列表组件ListView使用详解之动态加载或修改列表数据</a></li>
				<li><a href="http://www.manongjc.com/article/89593.html">Linux CentOS7 开启80，443端口外网访问权限</a></li>
				<li><a href="http://www.manongjc.com/article/89400.html">ubuntu安装chkconfig.deb系统服务管理工具</a></li>
				<li><a href="http://www.manongjc.com/article/89532.html">PHP中的 抽象类（abstract class）和 接口（interface）</a></li>
				<li><a href="http://www.manongjc.com/article/88645.html">微服务架构 - SpringCloud整合分布式服务跟踪zipkin</a></li>
				<li><a href="http://www.manongjc.com/article/88638.html">vue脚手架 build-config文件夹（跨域/打包）相关配置</a></li>
				<li><a href="http://www.manongjc.com/article/88487.html">JS如何给ul下的所有li绑定点击事件，点击使其弹出下标和内容</a></li>
				<div class="break"></div>
			</ul>
		</div>
		<div class="ads_in_hot_article">
			<a href="https://www.aliyun.com/minisite/goods?userCode=da65xs35" target="_BLANK"><img style="width:336px" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/336x280.jpg"></a>		</div>
		<div class="break"></div>
	</div>
</div>
				<div class="common">
		<div class="title-bar">本站知识点必读</div>
		<ul class="article-tutorial-list">
			<li><a href="http://www.manongjc.com/javascript/javascript_tutorial.html">JavaScript 教程</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_editor_tool.html">JavaScript 编辑工具</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_apply_html.html">JavaScript 与HTML</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_java.html">JavaScript 与Java</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_data_structure.html">JavaScript 数据结构</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_basic_data_types.html">JavaScript 基本数据类型</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_special_data_type.html">JavaScript 特殊数据类型</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_operator.html">JavaScript 运算符</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_typeof_operator.html">JavaScript typeof 运算符</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_expression.html">JavaScript 表达式</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_data_conversion.html">JavaScript 类型转换</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_basic_syntax.html">JavaScript 基本语法</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_comments.html">JavaScript 注释</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_basic_flow_control.html">Javascript 基本处理流程</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_select_structure.html">Javascript 选择结构</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_if_statements.html">Javascript if 语句</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_if_nested.html">Javascript if 语句的嵌套</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_switch.html">Javascript switch 语句</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_loop_structure.html">Javascript 循环结构</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_loop_example.html">Javascript 循环结构实例</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_break_continue.html">Javascript 跳转语句</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_flow_summary.html">Javascript 控制语句总结</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_function_brief.html">Javascript 函数介绍</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_function_define.html">Javascript 函数的定义</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_function_call.html">Javascript 函数调用</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_special_function.html">Javascript 几种特殊的函数</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_in_function.html">JavaScript 内置函数简介</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_eval.html">Javascript eval() 函数</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_isfinite.html">Javascript isFinite() 函数</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_isnan.html">Javascript isNaN() 函数</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_parseint_parsefloat.html">parseInt() 与 parseFloat()</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_escape_unescape.html">escape() 与 unescape()</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_string_object.html">Javascript 字符串介绍</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_length.html">Javascript length属性</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_string_functions.html">javascript 字符串函数</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_data_object_brief.html">Javascript 日期对象简介</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_data_use_example.html">Javascript 日期对象用途</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_date_attr_function.html">Date 对象属性和方法</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_what_is_array.html">Javascript 数组是什么</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_create_array.html">Javascript 创建数组</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_array_get_set_value.html">Javascript 数组赋值与取值</a></li>
			<li><a href="http://www.manongjc.com/javascript/js_array_method_attr.html">Javascript 数组属性和方法</a></li>
			<div class="break"></div>
		</ul>
		<div class="break"></div>
	</div>
		</div>
		<div class="right">
			<div id="outer_float_course" style="position: fixed; top: 0px;">
	<div id="float_course">
	<span>本网站教程列表：</span>
	<a href="http://www.manongjc.com/java/java_tutorial.html">JAVA教程</a>
	<a href="http://www.manongjc.com/java_example/java_environment_example.html">Java实例</a>
	<a href="http://www.manongjc.com/jsp/jsp_tutorial.html">JSP教程</a>
	<a href="http://www.manongjc.com/poi/poi_tutorial.html">Apache POI教程</a>
	<a href="http://www.manongjc.com/ejb/ejb_tutorial.html">EJB 教程</a>
	<a href="http://www.manongjc.com/jdbc/jdbc_tutorial.html">JDBC 教程</a>
	<a href="http://www.manongjc.com/spring/spring_tutorial.html">Spring 教程</a>
	<a href="http://www.manongjc.com/php/php_tutorial.html">PHP教程</a>
	<a href="http://www.manongjc.com/codeigniter/ci_tutorial.html">Codeigniter教程</a>
	<a href="http://www.manongjc.com/mysql/mysql_tutorial.html">MYSQL 教程</a>
	<a href="http://www.manongjc.com/mysql_basic/mysql-tutorial-basic.html">Mysql 基础教程</a>
	<a href="http://www.manongjc.com/sql/sql_tutorial.html">SQL 教程</a>
	<a href="http://www.manongjc.com/oracle/pl-sql-tutorial.html">PL/SQL教程</a>
	<a href="http://www.manongjc.com/sqlite/sqlite_tutorial.html">SQLite 教程</a>
	<a href="http://www.manongjc.com/redis/redis_tutorial.html">Redis 教程</a>
	<a href="http://www.manongjc.com/html/html_tutorial.html">HTML 教程</a>
	<a href="http://www.manongjc.com/css/css_tutorial.html">CSS教程</a>
	<a href="http://www.manongjc.com/css3/css3_tutorial.html">CSS3 教程</a>
	<a href="http://www.manongjc.com/cssref/css_reference.html">CSS 参考手册</a>
	<a href="http://www.manongjc.com/javascript/javascript_tutorial.html">Javascript 教程</a>
	<a href="http://www.manongjc.com/bootstrap/bootstrap_tutorial.html">Bootstrap 教程</a>
	<a href="http://www.manongjc.com/sass/sass_tutorial.html">Sass 教程</a>
	</div>
</div>

			<!--<div class="common">
	<div class="title-bar">推荐阅读</div>
	<ul class="caption_ul">
		<li><a href="/article/86.html"><img src="/images/css-border-1.png" /></a><a href="/article/86.html"><div class="caption"><span>css 巧用border属性制作各种图形(矩形，梯形，三角形)</span></div></a></li>
		<li><a href="/detail/25-tibllmhoyzxgodj.html"><img src="http://www.baizhud.com/images/6-1ZR9152K9621.gif" /></a><a href="/detail/25-tibllmhoyzxgodj.html"><div class="caption"><span>JavaScript break 语句</span></div></a></li>
		<li><a href="/article/333.html"><img src="/images/jquery_datatable_2.jpg" /></a><a href="/article/333.html"><div class="caption"><span>jQuery datatables 表格插件介绍</span></div></a></li>
	</ul>
</div>-->
			<div id="article_right_top"><!-- article_right_top -->
<ins class="adsbygoogle common_ad_class" style="display: inline-block; width: 300px; height: 250px;" data-ad-client="ca-pub-9401041403549801" data-ad-slot="1376902777" data-adsbygoogle-status="done" data-ad-status="filled"><ins id="aswift_2_expand" style="display:inline-table;border:none;height:250px;margin:0;padding:0;position:relative;visibility:visible;width:300px;background-color:transparent;" tabindex="0" title="Advertisement" aria-label="Advertisement"><ins id="aswift_2_anchor" style="display:block;border:none;height:250px;margin:0;padding:0;position:relative;visibility:visible;width:300px;background-color:transparent;"><iframe id="aswift_2" name="aswift_2" style="left:0;position:absolute;top:0;border:0;width:300px;height:250px;" sandbox="allow-forms allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-top-navigation-by-user-activation" width="300" height="250" frameborder="0" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/ads(2).html" marginwidth="0" marginheight="0" vspace="0" hspace="0" allowtransparency="true" scrolling="no" allowfullscreen="true" data-google-container-id="a!3" data-google-query-id="CPHo5JvY4fICFYqOSwUd_PIEQw" data-load-complete="true"></iframe></ins></ins></ins></div>
<script>
if(document.getElementById("article_right_top").offsetParent !== null){
	(adsbygoogle = window.adsbygoogle || []).push({});
}
</script>			<div class="common first-box">
				<div class="title-bar">
					<a href="http://www.manongjc.com/category/8.html">最近更新</a>
				</div>
				<ul>
					<li><a title="win10 在文件路径下打开命令行窗口" href="http://www.manongjc.com/detail/24-bshgntabohablow.html">win10 在文件路径下打开命令行窗口</a></li>
						<li><a title="powerDisigner使用" href="http://www.manongjc.com/detail/24-cxglcwqefxomtgi.html">powerDisigner使用</a></li>
						<li><a title="Spring Security 中 CSRF 防御源码解析" href="http://www.manongjc.com/detail/24-jiaqonkopzwqahe.html">Spring Security 中 CSRF 防御源码解析</a></li>
						<li><a title="SQL实战 6.对于employees表中，给出奇数行的first_name" href="http://www.manongjc.com/detail/24-wkbdvezmifjwsdd.html">SQL实战 6.对于employees表中，给出奇数行的first_name</a></li>
						<li><a title="vscode 自动保存格式化 c++ 的时候总是不正确换行" href="http://www.manongjc.com/detail/24-amlbdapoofnlvnt.html">vscode 自动保存格式化 c++ 的时候总是不正确换行</a></li>
						<li><a title="C# 文件目录操作" href="http://www.manongjc.com/detail/24-zkdbnzfnlnsvkky.html">C# 文件目录操作</a></li>
						<li><a title="go http rpc" href="http://www.manongjc.com/detail/24-nllznpzcdohqzde.html">go http rpc</a></li>
						<li><a title="java面向对象(一)" href="http://www.manongjc.com/detail/24-tdpxecwkqiyjdjt.html">java面向对象(一)</a></li>
						<li><a title="N1BOOK SQL注入-1 ---BUUCTF" href="http://www.manongjc.com/detail/24-cwfnwscvvgjcwjg.html">N1BOOK SQL注入-1 ---BUUCTF</a></li>
						<li><a title="java面向对象(二)" href="http://www.manongjc.com/detail/24-ojcwgyootbmnwfj.html">java面向对象(二)</a></li>
						<li><a title="Variable" href="http://www.manongjc.com/detail/24-iewjgvvkgnvcrgn.html">Variable</a></li>
						<li><a title="vscode pylint 出现 Module ‘numpy‘ has no ‘__‘ member (no-member)问题解决办法" href="http://www.manongjc.com/detail/24-rvxuretgtmigzfl.html">vscode pylint 出现 Module ‘numpy‘ has no ‘__‘ member (no-member)问题解决办法</a></li>
						<li><a title="windows 下后台开机自启动 frpc" href="http://www.manongjc.com/detail/24-pofgvckegswwgrk.html">windows 下后台开机自启动 frpc</a></li>
						<li><a title="Linux内核中root_domain的建立与初始化" href="http://www.manongjc.com/detail/24-nivjwbehthretci.html">Linux内核中root_domain的建立与初始化</a></li>
						<li><a title="98. 验证二叉搜索树" href="http://www.manongjc.com/detail/24-klccetvjttkkcpc.html">98. 验证二叉搜索树</a></li>
										</ul>
			</div>
            <div id="outer-ad-right-sticky-article"><div id="article_right_bottom" style="position: fixed; top: 0px;"><!-- article_right_bottom -->
<ins class="adsbygoogle common_ad_class" style="display: inline-block; width: 300px; height: 600px;" data-ad-client="ca-pub-9401041403549801" data-ad-slot="5807102372" data-adsbygoogle-status="done" data-ad-status="filled"><ins id="aswift_3_expand" style="display:inline-table;border:none;height:600px;margin:0;padding:0;position:relative;visibility:visible;width:300px;background-color:transparent;" tabindex="0" title="Advertisement" aria-label="Advertisement"><ins id="aswift_3_anchor" style="display:block;border:none;height:600px;margin:0;padding:0;position:relative;visibility:visible;width:300px;background-color:transparent;"><iframe id="aswift_3" name="aswift_3" style="left:0;position:absolute;top:0;border:0;width:300px;height:600px;" sandbox="allow-forms allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-top-navigation-by-user-activation" width="300" height="600" frameborder="0" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/ads(3).html" marginwidth="0" marginheight="0" vspace="0" hspace="0" allowtransparency="true" scrolling="no" allowfullscreen="true" data-google-container-id="a!4" data-google-query-id="CMny5JvY4fICFdpxfQodauEDEQ" data-load-complete="true"></iframe></ins></ins></ins></div></div>
<script>
if(document.getElementById("article_right_bottom").offsetParent !== null){
	(adsbygoogle = window.adsbygoogle || []).push({});
}
</script>		</div>
	</div>
</div>
<div class="footer">
	<div class="copyright">Copyright (C) 2015 www.manongjc.com, All Rights Reserved. 版权所有  码农教程<br>
	</div>
</div>
<script>
<!--百度流量统计-->
var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "//hm.baidu.com/hm.js?94faa21dbcd89dc3dccb1b3e4bb2e254";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script>
<!--51la流量统计-->
<script type="text/javascript" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/f(5).txt"></script>
<!--百度自动推送-->
<script>
(function(){
    var bp = document.createElement('script');
    var curProtocol = window.location.protocol.split(':')[0];
    if (curProtocol === 'https') {
        bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
    }
    else {
        bp.src = 'http://push.zhanzhang.baidu.com/push.js';
    }
    var s = document.getElementsByTagName("script")[0];
    s.parentNode.insertBefore(bp, s);
})();
</script>

<ins class="adsbygoogle adsbygoogle-noablate" data-adsbygoogle-status="done" style="display: none !important;" data-ad-status="unfilled"><ins id="aswift_4_expand" style="display:inline-table;border:none;height:0px;margin:0;padding:0;position:relative;visibility:visible;width:0px;background-color:transparent;" tabindex="0" title="Advertisement" aria-label="Advertisement"><ins id="aswift_4_anchor" style="display:block;border:none;height:0px;margin:0;padding:0;position:relative;visibility:visible;width:0px;background-color:transparent;"><iframe id="aswift_4" name="aswift_4" style="left:0;position:absolute;top:0;border:0;width:undefinedpx;height:undefinedpx;" sandbox="allow-forms allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-top-navigation-by-user-activation" frameborder="0" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/ads(4).html" marginwidth="0" marginheight="0" vspace="0" hspace="0" allowtransparency="true" scrolling="no" allowfullscreen="true" data-google-container-id="a!5" data-load-complete="true"></iframe></ins></ins></ins><iframe id="google_osd_static_frame_6972043818220" name="google_osd_static_frame" style="display: none; width: 0px; height: 0px;" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/saved_resource.html"></iframe><iframe src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/aframe.html" width="0" height="0" style="display: none;"></iframe></body><iframe id="google_esf" name="google_esf" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/zrt_lookup.html" style="display: none;"></iframe><ins class="adsbygoogle adsbygoogle-noablate" data-adsbygoogle-status="done" style="display: none !important; width: 100vw !important; height: 100vh !important; inset: 0px auto auto 0px !important; clear: none !important; float: none !important; margin: 0px !important; max-height: none !important; max-width: none !important; opacity: 1 !important; overflow: visible !important; padding: 0px !important; position: fixed !important; vertical-align: baseline !important; visibility: visible !important; z-index: 2147483647 !important; background-color: transparent;" aria-hidden="true" data-ad-status="filled" data-vignette-loaded="true"><ins id="aswift_5_expand" style="display: inline-table !important; border: none !important; height: 100vh !important; margin: 0px !important; padding: 0px !important; position: relative !important; visibility: visible !important; width: 100vw !important; background-color: transparent !important; inset: auto !important; clear: none !important; float: none !important; max-height: none !important; max-width: none !important; opacity: 1 !important; overflow: visible !important; vertical-align: baseline !important; z-index: auto !important;" tabindex="0" title="Advertisement" aria-label="Advertisement"><ins id="aswift_5_anchor" style="display: block !important; border: none !important; height: 100vh !important; margin: 0px !important; padding: 0px !important; position: relative !important; visibility: visible !important; width: 100vw !important; background-color: transparent !important; inset: auto !important; clear: none !important; float: none !important; max-height: none !important; max-width: none !important; opacity: 1 !important; overflow: visible !important; vertical-align: baseline !important; z-index: auto !important;"><iframe id="aswift_5" name="" sandbox="allow-forms allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-top-navigation-by-user-activation" width="" height="" frameborder="0" src="./在scrapy简单使用代理IP和随机的User-Agent - 码农教程_files/zrt_lookup(1).html" marginwidth="0" marginheight="0" vspace="0" hspace="0" allowtransparency="true" scrolling="no" allowfullscreen="true" style="width: 100vw !important; height: 100vh !important; inset: 0px auto auto 0px !important; position: absolute !important; clear: none !important; display: inline !important; float: none !important; margin: 0px !important; max-height: none !important; max-width: none !important; opacity: 1 !important; overflow: visible !important; padding: 0px !important; vertical-align: baseline !important; visibility: visible !important; z-index: auto !important;" data-google-container-id="a!6" data-google-query-id="COS14pvY4fICFcGfSwUdv7oDRQ" data-load-complete="true"></iframe></ins></ins></ins></html>